const playwright = require('playwright');
const TurndownService = require('turndown');
const path = require('path');
const fs = require('fs').promises;
const UrlNormalizer = require('../tools/UrlNormalizer');
const CaptureManager = require('./CaptureManager');
const DomOperations = require('./DomOperations');
const crypto = require('crypto');

/**
 * WebScraper 类 - 网页抓取工具
 * 提供灵活的网页内容抓取功能，支持自定义选择器和操作
 */
class WebScraper {
    /**
     * @param {Object} options - 配置选项
     * @param {Object} options.browser - 浏览器实例
     * @param {Object} options.captureManager - 抓取管理器实例
     * @param {Object} options.templateVars - 模板变量，用于替换配置中的模板
     */
    constructor(options = {}) {
        this.options = {
            headless: true,
            webDriver: null,
            configPath: path.join(__dirname, '..', 'config', 'sites.json'),
            ...options
        };

        // 加载配置文件
        this.loadConfig();

        // 初始化URL规范化工具
        this.urlNormalizer = new UrlNormalizer(options.urlNormalization);
        
        // 初始化内容管理器
        this.captureManager = new CaptureManager(options.captureManager);
        
        // 初始化Markdown转换器
        this.turndownService = new TurndownService({
            headingStyle: 'atx',
            codeBlockStyle: 'fenced'
        });

        this.browser = null;
        this.context = null;
        this.page = null;
        this.domOps = null;
        // 获取模板变量
        this.templateVars = options.templateVars || {};
    }

    /**
     * 加载配置文件
     */
    async loadConfig() {
        try {
            const configPath = path.join(__dirname, '../config/sites.json');
            // console.log('加载配置文件:', configPath);
            
            const configContent = await fs.readFile(configPath, 'utf-8');
            // console.log('配置文件内容:', configContent);
            
            this.siteConfigs = JSON.parse(configContent);
            // console.log('解析后的配置:', this.siteConfigs);
        } catch (error) {
            console.error('加载配置失败:', error);
            throw error;
        }
    }

    /**
     * 替换配置中的模板变量
     * @param {Object} config - 配置对象
     * @returns {Object} - 替换后的配置对象
     */
    replaceTemplateVars(config) {
        const stringified = JSON.stringify(config);
        const replaced = stringified.replace(/\${(\w+)}/g, (match, key) => {
            if (this.templateVars.hasOwnProperty(key)) {
                return this.templateVars[key];
            }
            console.warn(`模板变量 "${key}" 未定义`);
            return match;
        });
        return JSON.parse(replaced);
    }

    /**
     * 获取站点配置
     * @param {string} url - 目标URL
     * @returns {Object} - 站点配置
     */
    getSiteConfig(url) {
        
        // 获取网站特定配置
        const siteConfig = Object.entries(this.siteConfigs).find(([pattern]) => {
            if (pattern === 'default') return false;
            return url.includes(pattern);
        });

        if (siteConfig) {
            console.log('找到配置:', siteConfig[0]);
            console.log('找到配置:整体', siteConfig);
            // console log 输出 -> 找到配置: www.aplusa-online.com/vis/v1/en/exhprofiles
            // 替换模板变量
            const config = this.replaceTemplateVars(siteConfig[1]);
            return {
                ...this.siteConfigs.default,
                ...config
            };
        }

        console.log('使用默认配置');
        return this.siteConfigs.default;
    }

    /**
     * 获取特定网站的选择器配置
     * @param {string} hostname - 网站域名
     * @returns {Object} 选择器配置
     */
    getSelectors(hostname) {
        // 如果有特定网站的配置，使用它
        if (this.options.siteSelectors[hostname]) {
            return {
                ...this.options.selectors,
                ...this.options.siteSelectors[hostname]
            };
        }
        // 否则返回默认配置
        return this.options.selectors;
    }

    /**
     * 抓取网页内容
     * @param {string} url - 目标URL
     * @param {Object} options - 抓取选项
     * @returns {Promise<Object>} 抓取结果
     */
    async scrape(url, options = {}) {
        try {
            if (!this.browser) {
                await this.initBrowser();
            }

            // 获取配置
            const siteConfig = this.getSiteConfig(url);
            const finalOptions = {
                ...siteConfig,
                ...options
            };

            // 导航到页面
            await this.page.goto(url);
            console.log('页面加载完成:', url);

            // 执行所有动作并收集内容
            const scrapedData = {};
            if (finalOptions.actions) {
                for (const action of finalOptions.actions) {
                    console.log('执行动作:', action.type);
                    const actionResult = await this.executeAction(action);
                    if (actionResult) {
                        console.log('动作结果:', actionResult);
                        Object.assign(scrapedData, actionResult);
                    }
                }
            }

            // 获取主要内容
            if (finalOptions.content) {
                const contentElement = await this.page.$(finalOptions.content);
                if (contentElement) {
                    const html = await contentElement.innerHTML();
                    scrapedData.content = this.turndownService.turndown(html);
                    
                    // 下载并替换图片链接
                    scrapedData.content = await this.downloadImages(url, scrapedData.content);
                    // 下载并替换视频链接
                    scrapedData.content = await this.downloadVideos(url, scrapedData.content);
                }
            }

            // 处理扩展数据
            if (finalOptions.extendData) {
                const extendData = this.replaceTemplateVars(finalOptions.extendData);
                Object.assign(scrapedData, extendData);
            }

            // 保存抓取结果
            const normalizedUrl = this.urlNormalizer.normalize(url);
            const captureData = {
                url: normalizedUrl,
                ...scrapedData,
                timestamp: new Date().toISOString()
            };

            console.log('抓取结果:', captureData);

            // 保存到文件系统
            await this.captureManager.saveCapture(normalizedUrl, captureData);

            return captureData;
        } catch (error) {
            console.error('抓取失败:', error);
            throw error;
        }
    }

    /**
     * 执行动作
     * @param {Object} action - 动作配置
     * @returns {Promise<any>} 动作结果
     */
    async executeAction(action) {
        try {
            switch (action.type) {
                case 'wait':
                    if(action.selector){
                        await this.page.waitForSelector(action.selector, {
                            timeout: action.waitTime || 2000
                        });
                    }else{
                        await this.page.waitForTimeout(action.waitTime || 2000);
                    }
                    
                    break;
                    
                case 'scroll':
                    await this.page.evaluate(() => {
                        window.scrollTo(0, document.body.scrollHeight);
                    });
                    await this.page.waitForTimeout(1000); // 等待滚动完成
                    break;
                    
                case 'click':
                    const elementExists = await this.domOps.checkElementExists(action.selector);
                    if (elementExists) {
                        await this.page.click(action.selector);
                    } else {
                        console.log(`Element not found for click action: ${action.selector}`);
                    }
                    break;
                    
                case 'remove':
                    for (const selector of action.selectors) {
                        await this.domOps.handleRemove(selector);
                    }
                    break;
                    
                case 'selectors':
                    const content = {};
                    for (const [key, selectorConfig] of Object.entries(action.rules)) {
                        try {
                            const selector = selectorConfig.selector;
                            const attr = selectorConfig.attr || 'text';
                            const multi = selectorConfig.multi || false;

                            if (multi) {
                                // 获取所有匹配的元素
                                const elements = await this.page.$$(selector);
                                console.log(`多节点匹配：Found ${elements.length} elements matching ${selector}`);
                                content[key] = await Promise.all(
                                    elements.map(async element => {
                                        const value = await this.domOps.getElementContent(element, attr);
                                        return value !== null ? value : null;
                                    })
                                );
                                // 过滤掉 null 值
                                content[key] = content[key].filter(value => value !== null);
                            } else {
                                var isExist = await this.domOps.checkElementExists(selector);
                                if (isExist) {
                                    const element = await this.page.$(selector);
                                    content[key] = await this.domOps.getElementContent(element, attr);
                                }
                            }
                        } catch (error) {
                            console.warn(`Failed to extract ${key}:`, error);
                            content[key] = multi ? [] : null;
                        }
                    }
                    return content;
                    
                case 'screenshot':
                    return await this.executeScreenshotAction(action);

                case 'clickByText':
                    try {
                        const element = await this.page.getByText(action.selector).first();
                        const isVisible = await element.isVisible().catch(() => false);

                        if (isVisible) {
                            await element.click();
                            console.log(`已点击文本元素: "${action.selector}"`);
                        } else {
                            console.log(`未找到可点击的文本元素: "${action.selector}"`);
                        }
                    } catch (error) {
                        console.log(`点击文本元素失败: "${action.selector}"`, error);
                    }
                    break;
                    
                default:
                    console.warn(`Unknown action type: ${action.type}`);
            }
        } catch (error) {
            console.error(`Failed to execute action ${action.type}:`, error);
            throw error;
        }
    }

    /**
     * 执行截图动作
     * @param {Object} action - 动作配置
     * @returns {Promise<void>}
     */
    async executeScreenshotAction(action) {
        try {
            const { selector, waitTime = 1000, filename } = action;
            
            // 等待指定时间
            if (waitTime) {
                await this.page.waitForTimeout(waitTime);
            }

            // 生成截图文件名
            const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
            const screenshotName = filename 
                ? `${filename}.png`
                : `screenshot-${timestamp}.png`;

            if (selector) {
                // 如果指定了选择器，截取特定元素
                const element = await this.page.$(selector);
                if (element) {
                    const normalizedUrl = this.urlNormalizer.normalize(await this.page.url());
                    await this.captureManager.saveResource(
                        normalizedUrl,
                        screenshotName,
                        await element.screenshot(),
                        'assets'
                    );
                    console.log(`元素截图已保存: ${screenshotName}`);
                } else {
                    console.warn(`未找到指定元素: ${selector}`);
                }
            } else {
                // 如果没有指定选择器，截取整个页面
                const normalizedUrl = this.urlNormalizer.normalize(await this.page.url());
                await this.captureManager.saveResource(
                    normalizedUrl,
                    screenshotName,
                    await this.page.screenshot({ fullPage: true }),
                    'assets'
                );
                console.log(`页面截图已保存: ${screenshotName}`);
            }

            // 更新assets.json
            const assetsMap = await this.loadAssetsMap(await this.page.url()) || {};
            assetsMap[screenshotName] = {
                fileName: screenshotName,
                type: 'screenshot',
                mimeType: 'image/png',
                selector: selector || 'fullpage',
                customName: !!filename,
                captureTime: new Date().toISOString()
            };

            const normalizedUrl = this.urlNormalizer.normalize(await this.page.url());
            await this.captureManager.saveResource(
                normalizedUrl,
                'assets.json',
                Buffer.from(JSON.stringify(assetsMap, null, 2), 'utf-8'),
                ''
            );

        } catch (error) {
            console.error('执行截图动作时出错:', error);
        }
    }

    /**
     * 下载内容中的图片
     * @param {string} url - 原始URL
     * @param {string} content - Markdown内容
     * @returns {Promise<string>} 替换图片链接后的内容
     */
    async downloadImages(url, content) {
        try {
            // 匹配Markdown中的图片链接
            const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
            let match;
            let newContent = content;
            const downloadedImages = new Map();
            const assetsMap = {};

            while ((match = imageRegex.exec(content)) !== null) {
                const [fullMatch, altText, imageUrl] = match;
                if (downloadedImages.has(imageUrl)) {
                    // 如果已经下载过这个图片，直接使用已下载的路径
                    newContent = newContent.replace(imageUrl, downloadedImages.get(imageUrl));
                    continue;
                }

                try {
                    // 使用 Playwright 的 fetch API 下载图片
                    const response = await this.context.request.get(imageUrl);
                    if (!response.ok()) {
                        console.error(`下载图片失败: ${imageUrl}, 状态码: ${response.status()}`);
                        continue;
                    }

                    // 获取图片数据
                    const imageBuffer = await response.body();
                    
                    // 生成图片文件名（使用URL的hash）
                    const imageExt = imageUrl.split('.').pop().split('?')[0] || 'jpg';
                    const imageName = `${crypto.createHash('md5').update(imageUrl).digest('hex')}.${imageExt}`;

                    // 判断资源类型
                    const mimeType = response.headers()['content-type'] || '';
                    let assetType = 'unknown';
                    if (mimeType.startsWith('image/')) {
                        assetType = 'image';
                    } else if (mimeType.startsWith('video/')) {
                        assetType = 'video';
                    } else if (mimeType.startsWith('audio/')) {
                        assetType = 'audio';
                    }

                    // 保存图片
                    const normalizedUrl = this.urlNormalizer.normalize(url);
                    const savedPath = await this.captureManager.saveResource(
                        normalizedUrl,
                        imageName,
                        imageBuffer,
                        'assets'
                    );

                    // 获取相对路径
                    const relativePath = path.relative(
                        path.dirname(this.captureManager.getCaptureDir(normalizedUrl)),
                        savedPath
                    ).replace(/\\/g, '/');

                    // 更新内容中的图片链接
                    newContent = newContent.replace(imageUrl, relativePath);
                    downloadedImages.set(imageUrl, relativePath);

                    // 记录资源映射
                    assetsMap[imageName] = {
                        originalUrl: imageUrl,
                        fileName: imageName,
                        altText: altText || '',
                        type: assetType,
                        mimeType: mimeType,
                        size: imageBuffer.length,
                        downloadTime: new Date().toISOString()
                    };

                } catch (error) {
                    console.error(`处理图片时出错: ${imageUrl}`, error);
                }
            }

            // 如果有下载的资源，保存资源映射文件
            if (Object.keys(assetsMap).length > 0) {
                const normalizedUrl = this.urlNormalizer.normalize(url);
                await this.captureManager.saveResource(
                    normalizedUrl,
                    'assets.json',
                    Buffer.from(JSON.stringify(assetsMap, null, 2), 'utf-8'),
                    '' // 空字符串表示保存在根目录
                );
            }

            return newContent;
        } catch (error) {
            console.error('下载图片过程中出错:', error);
            return content; // 如果出错，返回原始内容
        }
    }

    /**
     * 下载视频文件
     * @param {string} url - 原始URL
     * @param {string} content - HTML/Markdown内容
     * @returns {Promise<string>} 替换视频链接后的内容
     */
    async downloadVideos(url, content) {
        try {
            // 匹配HTML video标签中的视频链接
            const videoRegex = /<video[^>]*>(?:\s*<source[^>]*src="([^"]+)"[^>]*>)*[^<]*<\/video>/g;
            const sourceRegex = /src="([^"]+)"/;
            let match;
            let newContent = content;
            const downloadedVideos = new Map();
            const assetsMap = await this.loadAssetsMap(url) || {};

            while ((match = videoRegex.exec(content)) !== null) {
                const videoTag = match[0];
                const sourceMatch = videoTag.match(sourceRegex);
                if (!sourceMatch) continue;

                const videoUrl = sourceMatch[1];
                if (downloadedVideos.has(videoUrl)) {
                    // 如果已经下载过这个视频，直接使用已下载的路径
                    newContent = newContent.replace(videoUrl, downloadedVideos.get(videoUrl));
                    continue;
                }

                try {
                    console.log(`开始下载视频: ${videoUrl}`);
                    // 使用 Playwright 的 fetch API 下载视频
                    const response = await this.context.request.get(videoUrl);
                    if (!response.ok()) {
                        console.error(`下载视频失败: ${videoUrl}, 状态码: ${response.status()}`);
                        continue;
                    }

                    // 获取视频数据
                    const videoBuffer = await response.body();
                    
                    // 生成视频文件名
                    const videoExt = videoUrl.split('.').pop().split('?')[0] || 'mp4';
                    const videoName = `${crypto.createHash('md5').update(videoUrl).digest('hex')}.${videoExt}`;

                    // 获取MIME类型
                    const mimeType = response.headers()['content-type'] || '';

                    // 保存视频
                    const normalizedUrl = this.urlNormalizer.normalize(url);
                    const savedPath = await this.captureManager.saveResource(
                        normalizedUrl,
                        videoName,
                        videoBuffer,
                        'assets'
                    );

                    // 获取相对路径
                    const relativePath = path.relative(
                        path.dirname(this.captureManager.getCaptureDir(normalizedUrl)),
                        savedPath
                    ).replace(/\\/g, '/');

                    // 更新内容中的视频链接
                    newContent = newContent.replace(videoUrl, relativePath);
                    downloadedVideos.set(videoUrl, relativePath);

                    // 记录资源映射
                    assetsMap[videoName] = {
                        originalUrl: videoUrl,
                        fileName: videoName,
                        type: 'video',
                        mimeType: mimeType,
                        size: videoBuffer.length,
                        downloadTime: new Date().toISOString()
                    };

                    console.log(`视频下载完成: ${videoName}`);
                } catch (error) {
                    console.error(`处理视频时出错: ${videoUrl}`, error);
                }
            }

            // 如果有下载的资源，更新资源映射文件
            if (Object.keys(assetsMap).length > 0) {
                const normalizedUrl = this.urlNormalizer.normalize(url);
                await this.captureManager.saveResource(
                    normalizedUrl,
                    'assets.json',
                    Buffer.from(JSON.stringify(assetsMap, null, 2), 'utf-8'),
                    ''
                );
            }

            return newContent;
        } catch (error) {
            console.error('下载视频过程中出错:', error);
            return content;
        }
    }

    /**
     * 加载现有的assets.json文件
     * @param {string} url - 原始URL
     * @returns {Promise<Object>} assets映射对象
     */
    async loadAssetsMap(url) {
        try {
            const normalizedUrl = this.urlNormalizer.normalize(url);
            const assetsPath = path.join(
                this.captureManager.getCaptureDir(normalizedUrl),
                'assets.json'
            );
            
            try {
                const content = await fs.readFile(assetsPath, 'utf-8');
                return JSON.parse(content);
            } catch (err) {
                if (err.code === 'ENOENT') {
                    return {};
                }
                throw err;
            }
        } catch (error) {
            console.error('加载assets.json失败:', error);
            return {};
        }
    }

    /**
     * 设置模板变量
     * @param {Object} vars - 新的模板变量
     */
    setTemplateVars(vars) {
        this.templateVars = { ...this.templateVars, ...vars };
    }

    /**
     * 加载配置文件
     */
    async loadConfig() {
        try {
            const configPath = path.join(__dirname, '../config/sites.json');
            // console.log('加载配置文件:', configPath);
            
            const configContent = await fs.readFile(configPath, 'utf-8');
            // console.log('配置文件内容:', configContent);
            
            this.siteConfigs = JSON.parse(configContent);
            // console.log('解析后的配置:', this.siteConfigs);
        } catch (error) {
            console.error('加载配置失败:', error);
            throw error;
        }
    }

    /**
     * 初始化浏览器
     */
    async initBrowser() {
        try {
            // 确保配置已加载
            if (!this.siteConfigs) {
                await this.loadConfig();
            }

            if (this.options.webDriver && this.options.webDriver.endpoint) {
                console.log('正在连接到WebDriver:', this.options.webDriver.endpoint);
                this.browser = await playwright.chromium.connectOverCDP(this.options.webDriver.endpoint);
                this.context = await this.browser.contexts()[0];
            } else {
                this.browser = await playwright.chromium.launch({
                    headless: this.options.headless
                });
                this.context = await this.browser.newContext();
            }
            console.log('浏览器初始化完成');

            // 创建初始页面
            await this.createNewPage();
            
            // 初始化DOM操作工具
            this.domOps = new DomOperations(this.page);
        } catch (error) {
            console.error('浏览器初始化失败:', error);
            throw error;
        }
    }

    /**
     * 创建新的页面
     * @returns {Promise<void>}
     */
    async createNewPage() {
        if (!this.context) {
            await this.initBrowser();
        }
        this.page = await this.context.newPage();
        this.domOps = new DomOperations(this.page);
    }

    /**
     * 关闭当前页面并创建新的页面
     * @returns {Promise<void>}
     */
    async closePage() {
        if (this.page) {
            await this.page.close();
            this.page = null;
        }
        // 创建新的页面
        await this.createNewPage();
    }

    /**
     * 关闭浏览器实例
     */
    async close() {
        if (this.page) {
            await this.page.close();
        }
        if (this.browser) {
            await this.browser.close();
        }
    }
}

module.exports = WebScraper;
